/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package tokenizer;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Set;
import java.util.StringTokenizer;

/**
 *
 * @author Zakiy Firdaus
 */
public class Tokenizer {
    public Tokenizer(){
        
    }
    
    /**
     * melakukan split string terhadap pemisahnya
     * fungsi ini juga mengatasi masalah penggunaan simbol (-) dan (')
     * @param content string yang menjadi masukan untuk dicari token-tokennya
     * @param rawcontent string awal yang belum dihilangkan tag htmlnya
     * @param initialpos posisi awal string masukan terhadap posisinya dalam email
     * @return HashMap <String token, ArrayList<Integer> posisi-posisi token tersebut>
     */
    public HashMap termSplitter(String content, String rawcontent, int initialpos) {
        //mengenerate token-token yang dipisahkan oleh space, whitespace, punctuation marks, etc
        HashMap<String, ArrayList<Integer>> result = new HashMap<String, ArrayList<Integer>>();
        StringTokenizer termSplitter = new StringTokenizer(content, "\t\n\0,.:;?!| [](){}/\\\"^&=#");
        int curpos = 0;
        int termpos = 0;
        while (termSplitter.hasMoreElements()) {
            String thisterm = termSplitter.nextToken();
            termpos = rawcontent.indexOf(thisterm,curpos);
            curpos = termpos+1;
            termpos += initialpos;
            //menghapus (-)
            thisterm = thisterm.replaceAll("-","");
            //menghapus ('s) yang menerangkan kepemilikan
            if(thisterm.length()>=2){
                if(thisterm.substring(thisterm.length()-2).equals("'s")){
                    thisterm = thisterm.substring(0, thisterm.length()-2);
                }
            }
            //menghapus (')
            thisterm = thisterm.replaceAll("'","");
            if(!result.containsKey(thisterm)){
                ArrayList<Integer> listPos = new ArrayList<Integer>();
                listPos.add(termpos);
                result.put(thisterm,listPos);
            } else {
                ArrayList<Integer> listPos = new ArrayList<Integer>();
                listPos = result.get(thisterm);
                listPos.add(termpos);
                result.put(thisterm,listPos);
            }
        }
        return result;
    }
    
    /**
     * fungsi untuk menghilangkan tag HTML/XML dari suatu string
     * @param content string masukan
     * @return string masukan tanpa tag HTML/XML
     */
    public String removeHTMLTag(String content){
        int idxAwalTag = content.indexOf("<");
        int idxAkhirTag;
        while(idxAwalTag!=-1){
            idxAkhirTag = content.indexOf(">");
            while(idxAkhirTag<idxAwalTag&&idxAkhirTag!=-1){
                idxAkhirTag = content.indexOf(">", idxAkhirTag+1);
            }
            if(idxAkhirTag!=-1){
                if(idxAkhirTag >= content.length()-1){
                    content = content.substring(0, idxAwalTag);
                } else {
                    content = content.substring(0, idxAwalTag)+content.substring(idxAkhirTag+1);
                }

                idxAwalTag = content.indexOf("<");
            } else {
                idxAwalTag=-1;
            }
        }
        content = content.replaceAll("<", "");
        content = content.replaceAll(">", "");
        return content;
    }
    
    /**
     * fungsi lengkap dari proses tokenisasi
     * @param content string masukan
     * @param pos posisi awal string masukan terhadap posisinya dalam email
     * @return HashMap <String token, ArrayList<Integer> posisi-posisi token tersebut>
     */
    public HashMap tokenize(String content, int pos) {
        String content2="";
        content2 = removeHTMLTag(content);
        return termSplitter(content2, content, pos);
    }
    
    /**
     * prosedur untuk menguji keberjalanan tokenizer
     * @param args 
     */
    public static void main(String[] args) {
        Tokenizer tar = new Tokenizer();
        String s = "I didn't see Mary's book. Did O'donnel see that state-of-the-art book? The book's title is <Title><fontstyle = \"bold\">\"Introduction to>>> Information Retrieval\"</fontstyle></Title>";
        System.out.println("String awal: "+s);
        System.out.println("--");
        System.out.println("tag html dihapus:");
        String s2 = tar.removeHTMLTag(s);
        System.out.println(s2);
        System.out.println("--");
        HashMap<String, ArrayList<Integer>> token = new HashMap<String, ArrayList<Integer>>();
        token = tar.tokenize(s, 0);
        for(String key: token.keySet()){
            System.out.println(key+": "+token.get(key));
        }
    }
}
